{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## 缺失数据\n",
    "\n",
    "这个部分其实非常常用的，我之前做过一个简单的数据分析发现对缺失数据的处理必不可少\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [
    {
     "data": {
      "text/plain": "        SaleID    name   regDate  model  brand  bodyType  fuelType  gearbox  \\\n0            0     736  20040402   30.0      6       1.0       0.0      0.0   \n1            1    2262  20030301   40.0      1       2.0       0.0      0.0   \n2            2   14874  20040403  115.0     15       1.0       0.0      0.0   \n3            3   71865  19960908  109.0     10       0.0       0.0      1.0   \n4            4  111080  20120103  110.0      5       1.0       0.0      0.0   \n...        ...     ...       ...    ...    ...       ...       ...      ...   \n149995  149995  163978  20000607  121.0     10       4.0       0.0      1.0   \n149996  149996  184535  20091102  116.0     11       0.0       0.0      0.0   \n149997  149997  147587  20101003   60.0     11       1.0       1.0      0.0   \n149998  149998   45907  20060312   34.0     10       3.0       1.0      0.0   \n149999  149999  177672  19990204   19.0     28       6.0       0.0      1.0   \n\n        power  kilometer notRepairedDamage  regionCode  seller  offerType  \\\n0          60       12.5               0.0        1046       0          0   \n1           0       15.0                 -        4366       0          0   \n2         163       12.5               0.0        2806       0          0   \n3         193       15.0               0.0         434       0          0   \n4          68        5.0               0.0        6977       0          0   \n...       ...        ...               ...         ...     ...        ...   \n149995    163       15.0               0.0        4576       0          0   \n149996    125       10.0               0.0        2826       0          0   \n149997     90        6.0               0.0        3302       0          0   \n149998    156       15.0               0.0        1877       0          0   \n149999    193       12.5               0.0         235       0          0   \n\n        creatDate  price  \n0        20160404   1850  \n1        20160309   3600  \n2        20160402   6222  \n3        20160312   2400  \n4        20160313   5200  \n...           ...    ...  \n149995   20160327   5900  \n149996   20160312   9500  \n149997   20160328   7500  \n149998   20160401   4999  \n149999   20160305   4700  \n\n[150000 rows x 16 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>SaleID</th>\n      <th>name</th>\n      <th>regDate</th>\n      <th>model</th>\n      <th>brand</th>\n      <th>bodyType</th>\n      <th>fuelType</th>\n      <th>gearbox</th>\n      <th>power</th>\n      <th>kilometer</th>\n      <th>notRepairedDamage</th>\n      <th>regionCode</th>\n      <th>seller</th>\n      <th>offerType</th>\n      <th>creatDate</th>\n      <th>price</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>736</td>\n      <td>20040402</td>\n      <td>30.0</td>\n      <td>6</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>60</td>\n      <td>12.5</td>\n      <td>0.0</td>\n      <td>1046</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160404</td>\n      <td>1850</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>2262</td>\n      <td>20030301</td>\n      <td>40.0</td>\n      <td>1</td>\n      <td>2.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>-</td>\n      <td>4366</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160309</td>\n      <td>3600</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>14874</td>\n      <td>20040403</td>\n      <td>115.0</td>\n      <td>15</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>163</td>\n      <td>12.5</td>\n      <td>0.0</td>\n      <td>2806</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160402</td>\n      <td>6222</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>71865</td>\n      <td>19960908</td>\n      <td>109.0</td>\n      <td>10</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>193</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>434</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160312</td>\n      <td>2400</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>111080</td>\n      <td>20120103</td>\n      <td>110.0</td>\n      <td>5</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>68</td>\n      <td>5.0</td>\n      <td>0.0</td>\n      <td>6977</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160313</td>\n      <td>5200</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>149995</th>\n      <td>149995</td>\n      <td>163978</td>\n      <td>20000607</td>\n      <td>121.0</td>\n      <td>10</td>\n      <td>4.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>163</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>4576</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160327</td>\n      <td>5900</td>\n    </tr>\n    <tr>\n      <th>149996</th>\n      <td>149996</td>\n      <td>184535</td>\n      <td>20091102</td>\n      <td>116.0</td>\n      <td>11</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>125</td>\n      <td>10.0</td>\n      <td>0.0</td>\n      <td>2826</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160312</td>\n      <td>9500</td>\n    </tr>\n    <tr>\n      <th>149997</th>\n      <td>149997</td>\n      <td>147587</td>\n      <td>20101003</td>\n      <td>60.0</td>\n      <td>11</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>90</td>\n      <td>6.0</td>\n      <td>0.0</td>\n      <td>3302</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160328</td>\n      <td>7500</td>\n    </tr>\n    <tr>\n      <th>149998</th>\n      <td>149998</td>\n      <td>45907</td>\n      <td>20060312</td>\n      <td>34.0</td>\n      <td>10</td>\n      <td>3.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>156</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>1877</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160401</td>\n      <td>4999</td>\n    </tr>\n    <tr>\n      <th>149999</th>\n      <td>149999</td>\n      <td>177672</td>\n      <td>19990204</td>\n      <td>19.0</td>\n      <td>28</td>\n      <td>6.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>193</td>\n      <td>12.5</td>\n      <td>0.0</td>\n      <td>235</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160305</td>\n      <td>4700</td>\n    </tr>\n  </tbody>\n</table>\n<p>150000 rows × 16 columns</p>\n</div>"
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "data_csv = pd.read_csv('../二手车交易价格预测/data/used_car_train_20200313.csv',sep=' ')\n",
    "data_csv = data_csv.drop(data_csv.columns[16:],axis=1)\n",
    "data_csv"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 查看每列缺失的比例\n",
    "这个操作非常必要，并且很直观\n",
    "\n",
    "这里还是得强调一下关键参数：`axis`。这个参数现在发现基本贯穿了整个Pandas学习。我想了半天都没想明白该如何表述，直接上图吧：\n",
    "![axis](https://img-blog.csdnimg.cn/20200320204758144.jpg)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [
    {
     "data": {
      "text/plain": "SaleID               0.000000\nname                 0.000000\nregDate              0.000000\nmodel                0.000007\nbrand                0.000000\nbodyType             0.030040\nfuelType             0.057867\ngearbox              0.039873\npower                0.000000\nkilometer            0.000000\nnotRepairedDamage    0.000000\nregionCode           0.000000\nseller               0.000000\nofferType            0.000000\ncreatDate            0.000000\nprice                0.000000\ndtype: float64"
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看每一列的缺失比例（True）\n",
    "data_csv.isnull().mean(axis=0)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 至少有一个缺失\n",
    "又学到一个语法：`data_csv`是一个DataFrame，里面可以传递 True/False的**Series**。第一次知道这样的事情，一时难以接受。😒"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [
    {
     "data": {
      "text/plain": "0         False\n1         False\n2         False\n3         False\n4         False\n          ...  \n149995    False\n149996    False\n149997    False\n149998    False\n149999    False\nLength: 150000, dtype: bool"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_csv.isna().any(1)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [
    {
     "data": {
      "text/plain": "        SaleID    name   regDate  model  brand  bodyType  fuelType  gearbox  \\\n14          14    1896  20070009    1.0      0       NaN       NaN      0.0   \n21          21   12784  20021009    8.0      0       0.0       NaN      NaN   \n42          42   20694  19960009    0.0      0       4.0       NaN      0.0   \n45          45    8893  20010306   16.0     13       1.0       NaN      0.0   \n98          98   31752  20020011    1.0      1       2.0       NaN      1.0   \n...        ...     ...       ...    ...    ...       ...       ...      ...   \n149937  149937   93471  20071110    8.0      0       0.0       1.0      NaN   \n149957  149957    3075  20000312   73.0     14       NaN       NaN      0.0   \n149970  149970  102173  19970406   48.0     14       1.0       NaN      0.0   \n149972  149972  183896  20050004   41.0      6       NaN       NaN      NaN   \n149973  149973   17265  20051104   49.0      1       0.0       1.0      NaN   \n\n        power  kilometer notRepairedDamage  regionCode  seller  offerType  \\\n14          0       15.0                 -        3972       0          0   \n21        116       15.0                 -        1278       0          0   \n42         90       15.0               0.0        4825       0          0   \n45         60       15.0               0.0        1326       0          0   \n98          0       15.0               0.0        6792       0          0   \n...       ...        ...               ...         ...     ...        ...   \n149937      0       15.0               0.0        1128       0          0   \n149957    130       15.0                 -        4439       0          0   \n149970     45       15.0               0.0        2159       0          0   \n149972      0       15.0                 -        7078       0          0   \n149973      0       15.0               0.0        4379       0          0   \n\n        creatDate  price  \n14       20160402   6900  \n21       20160403   2800  \n42       20160328   1600  \n45       20160330    850  \n98       20160310   3000  \n...           ...    ...  \n149937   20160326   3100  \n149957   20160402    600  \n149970   20160407    900  \n149972   20160326    999  \n149973   20160316   3500  \n\n[14116 rows x 16 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>SaleID</th>\n      <th>name</th>\n      <th>regDate</th>\n      <th>model</th>\n      <th>brand</th>\n      <th>bodyType</th>\n      <th>fuelType</th>\n      <th>gearbox</th>\n      <th>power</th>\n      <th>kilometer</th>\n      <th>notRepairedDamage</th>\n      <th>regionCode</th>\n      <th>seller</th>\n      <th>offerType</th>\n      <th>creatDate</th>\n      <th>price</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>14</th>\n      <td>14</td>\n      <td>1896</td>\n      <td>20070009</td>\n      <td>1.0</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>-</td>\n      <td>3972</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160402</td>\n      <td>6900</td>\n    </tr>\n    <tr>\n      <th>21</th>\n      <td>21</td>\n      <td>12784</td>\n      <td>20021009</td>\n      <td>8.0</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>15.0</td>\n      <td>-</td>\n      <td>1278</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160403</td>\n      <td>2800</td>\n    </tr>\n    <tr>\n      <th>42</th>\n      <td>42</td>\n      <td>20694</td>\n      <td>19960009</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>4.0</td>\n      <td>NaN</td>\n      <td>0.0</td>\n      <td>90</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>4825</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160328</td>\n      <td>1600</td>\n    </tr>\n    <tr>\n      <th>45</th>\n      <td>45</td>\n      <td>8893</td>\n      <td>20010306</td>\n      <td>16.0</td>\n      <td>13</td>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>0.0</td>\n      <td>60</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>1326</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160330</td>\n      <td>850</td>\n    </tr>\n    <tr>\n      <th>98</th>\n      <td>98</td>\n      <td>31752</td>\n      <td>20020011</td>\n      <td>1.0</td>\n      <td>1</td>\n      <td>2.0</td>\n      <td>NaN</td>\n      <td>1.0</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>6792</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160310</td>\n      <td>3000</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>149937</th>\n      <td>149937</td>\n      <td>93471</td>\n      <td>20071110</td>\n      <td>8.0</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>1128</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160326</td>\n      <td>3100</td>\n    </tr>\n    <tr>\n      <th>149957</th>\n      <td>149957</td>\n      <td>3075</td>\n      <td>20000312</td>\n      <td>73.0</td>\n      <td>14</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>0.0</td>\n      <td>130</td>\n      <td>15.0</td>\n      <td>-</td>\n      <td>4439</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160402</td>\n      <td>600</td>\n    </tr>\n    <tr>\n      <th>149970</th>\n      <td>149970</td>\n      <td>102173</td>\n      <td>19970406</td>\n      <td>48.0</td>\n      <td>14</td>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>0.0</td>\n      <td>45</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>2159</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160407</td>\n      <td>900</td>\n    </tr>\n    <tr>\n      <th>149972</th>\n      <td>149972</td>\n      <td>183896</td>\n      <td>20050004</td>\n      <td>41.0</td>\n      <td>6</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>-</td>\n      <td>7078</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160326</td>\n      <td>999</td>\n    </tr>\n    <tr>\n      <th>149973</th>\n      <td>149973</td>\n      <td>17265</td>\n      <td>20051104</td>\n      <td>49.0</td>\n      <td>1</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>4379</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160316</td>\n      <td>3500</td>\n    </tr>\n  </tbody>\n</table>\n<p>14116 rows × 16 columns</p>\n</div>"
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#法一\n",
    "data_csv[data_csv.isna().any(1)]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [
    {
     "data": {
      "text/plain": "        SaleID    name   regDate  model  brand  bodyType  fuelType  gearbox  \\\n14          14    1896  20070009    1.0      0       NaN       NaN      0.0   \n21          21   12784  20021009    8.0      0       0.0       NaN      NaN   \n42          42   20694  19960009    0.0      0       4.0       NaN      0.0   \n45          45    8893  20010306   16.0     13       1.0       NaN      0.0   \n98          98   31752  20020011    1.0      1       2.0       NaN      1.0   \n...        ...     ...       ...    ...    ...       ...       ...      ...   \n149937  149937   93471  20071110    8.0      0       0.0       1.0      NaN   \n149957  149957    3075  20000312   73.0     14       NaN       NaN      0.0   \n149970  149970  102173  19970406   48.0     14       1.0       NaN      0.0   \n149972  149972  183896  20050004   41.0      6       NaN       NaN      NaN   \n149973  149973   17265  20051104   49.0      1       0.0       1.0      NaN   \n\n        power  kilometer notRepairedDamage  regionCode  seller  offerType  \\\n14          0       15.0                 -        3972       0          0   \n21        116       15.0                 -        1278       0          0   \n42         90       15.0               0.0        4825       0          0   \n45         60       15.0               0.0        1326       0          0   \n98          0       15.0               0.0        6792       0          0   \n...       ...        ...               ...         ...     ...        ...   \n149937      0       15.0               0.0        1128       0          0   \n149957    130       15.0                 -        4439       0          0   \n149970     45       15.0               0.0        2159       0          0   \n149972      0       15.0                 -        7078       0          0   \n149973      0       15.0               0.0        4379       0          0   \n\n        creatDate  price  \n14       20160402   6900  \n21       20160403   2800  \n42       20160328   1600  \n45       20160330    850  \n98       20160310   3000  \n...           ...    ...  \n149937   20160326   3100  \n149957   20160402    600  \n149970   20160407    900  \n149972   20160326    999  \n149973   20160316   3500  \n\n[14116 rows x 16 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>SaleID</th>\n      <th>name</th>\n      <th>regDate</th>\n      <th>model</th>\n      <th>brand</th>\n      <th>bodyType</th>\n      <th>fuelType</th>\n      <th>gearbox</th>\n      <th>power</th>\n      <th>kilometer</th>\n      <th>notRepairedDamage</th>\n      <th>regionCode</th>\n      <th>seller</th>\n      <th>offerType</th>\n      <th>creatDate</th>\n      <th>price</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>14</th>\n      <td>14</td>\n      <td>1896</td>\n      <td>20070009</td>\n      <td>1.0</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>-</td>\n      <td>3972</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160402</td>\n      <td>6900</td>\n    </tr>\n    <tr>\n      <th>21</th>\n      <td>21</td>\n      <td>12784</td>\n      <td>20021009</td>\n      <td>8.0</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>15.0</td>\n      <td>-</td>\n      <td>1278</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160403</td>\n      <td>2800</td>\n    </tr>\n    <tr>\n      <th>42</th>\n      <td>42</td>\n      <td>20694</td>\n      <td>19960009</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>4.0</td>\n      <td>NaN</td>\n      <td>0.0</td>\n      <td>90</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>4825</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160328</td>\n      <td>1600</td>\n    </tr>\n    <tr>\n      <th>45</th>\n      <td>45</td>\n      <td>8893</td>\n      <td>20010306</td>\n      <td>16.0</td>\n      <td>13</td>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>0.0</td>\n      <td>60</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>1326</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160330</td>\n      <td>850</td>\n    </tr>\n    <tr>\n      <th>98</th>\n      <td>98</td>\n      <td>31752</td>\n      <td>20020011</td>\n      <td>1.0</td>\n      <td>1</td>\n      <td>2.0</td>\n      <td>NaN</td>\n      <td>1.0</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>6792</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160310</td>\n      <td>3000</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>149937</th>\n      <td>149937</td>\n      <td>93471</td>\n      <td>20071110</td>\n      <td>8.0</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>1128</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160326</td>\n      <td>3100</td>\n    </tr>\n    <tr>\n      <th>149957</th>\n      <td>149957</td>\n      <td>3075</td>\n      <td>20000312</td>\n      <td>73.0</td>\n      <td>14</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>0.0</td>\n      <td>130</td>\n      <td>15.0</td>\n      <td>-</td>\n      <td>4439</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160402</td>\n      <td>600</td>\n    </tr>\n    <tr>\n      <th>149970</th>\n      <td>149970</td>\n      <td>102173</td>\n      <td>19970406</td>\n      <td>48.0</td>\n      <td>14</td>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>0.0</td>\n      <td>45</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>2159</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160407</td>\n      <td>900</td>\n    </tr>\n    <tr>\n      <th>149972</th>\n      <td>149972</td>\n      <td>183896</td>\n      <td>20050004</td>\n      <td>41.0</td>\n      <td>6</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>-</td>\n      <td>7078</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160326</td>\n      <td>999</td>\n    </tr>\n    <tr>\n      <th>149973</th>\n      <td>149973</td>\n      <td>17265</td>\n      <td>20051104</td>\n      <td>49.0</td>\n      <td>1</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>4379</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160316</td>\n      <td>3500</td>\n    </tr>\n  </tbody>\n</table>\n<p>14116 rows × 16 columns</p>\n</div>"
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#法二，这种方式好理解些\n",
    "data_csv.loc[data_csv.isnull().any(1)]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 缺失信息的删除\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### 先尝试不用dropna()的方式"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [
    {
     "data": {
      "text/plain": "        SaleID    name   regDate  model  brand  bodyType  fuelType  gearbox  \\\n0            0     736  20040402   30.0      6       1.0       0.0      0.0   \n1            1    2262  20030301   40.0      1       2.0       0.0      0.0   \n2            2   14874  20040403  115.0     15       1.0       0.0      0.0   \n3            3   71865  19960908  109.0     10       0.0       0.0      1.0   \n4            4  111080  20120103  110.0      5       1.0       0.0      0.0   \n...        ...     ...       ...    ...    ...       ...       ...      ...   \n149995  149995  163978  20000607  121.0     10       4.0       0.0      1.0   \n149996  149996  184535  20091102  116.0     11       0.0       0.0      0.0   \n149997  149997  147587  20101003   60.0     11       1.0       1.0      0.0   \n149998  149998   45907  20060312   34.0     10       3.0       1.0      0.0   \n149999  149999  177672  19990204   19.0     28       6.0       0.0      1.0   \n\n        power  kilometer notRepairedDamage  regionCode  seller  offerType  \\\n0          60       12.5               0.0        1046       0          0   \n1           0       15.0                 -        4366       0          0   \n2         163       12.5               0.0        2806       0          0   \n3         193       15.0               0.0         434       0          0   \n4          68        5.0               0.0        6977       0          0   \n...       ...        ...               ...         ...     ...        ...   \n149995    163       15.0               0.0        4576       0          0   \n149996    125       10.0               0.0        2826       0          0   \n149997     90        6.0               0.0        3302       0          0   \n149998    156       15.0               0.0        1877       0          0   \n149999    193       12.5               0.0         235       0          0   \n\n        creatDate  price  \n0        20160404   1850  \n1        20160309   3600  \n2        20160402   6222  \n3        20160312   2400  \n4        20160313   5200  \n...           ...    ...  \n149995   20160327   5900  \n149996   20160312   9500  \n149997   20160328   7500  \n149998   20160401   4999  \n149999   20160305   4700  \n\n[135884 rows x 16 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>SaleID</th>\n      <th>name</th>\n      <th>regDate</th>\n      <th>model</th>\n      <th>brand</th>\n      <th>bodyType</th>\n      <th>fuelType</th>\n      <th>gearbox</th>\n      <th>power</th>\n      <th>kilometer</th>\n      <th>notRepairedDamage</th>\n      <th>regionCode</th>\n      <th>seller</th>\n      <th>offerType</th>\n      <th>creatDate</th>\n      <th>price</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>736</td>\n      <td>20040402</td>\n      <td>30.0</td>\n      <td>6</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>60</td>\n      <td>12.5</td>\n      <td>0.0</td>\n      <td>1046</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160404</td>\n      <td>1850</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>2262</td>\n      <td>20030301</td>\n      <td>40.0</td>\n      <td>1</td>\n      <td>2.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>-</td>\n      <td>4366</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160309</td>\n      <td>3600</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>14874</td>\n      <td>20040403</td>\n      <td>115.0</td>\n      <td>15</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>163</td>\n      <td>12.5</td>\n      <td>0.0</td>\n      <td>2806</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160402</td>\n      <td>6222</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>71865</td>\n      <td>19960908</td>\n      <td>109.0</td>\n      <td>10</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>193</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>434</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160312</td>\n      <td>2400</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>111080</td>\n      <td>20120103</td>\n      <td>110.0</td>\n      <td>5</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>68</td>\n      <td>5.0</td>\n      <td>0.0</td>\n      <td>6977</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160313</td>\n      <td>5200</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>149995</th>\n      <td>149995</td>\n      <td>163978</td>\n      <td>20000607</td>\n      <td>121.0</td>\n      <td>10</td>\n      <td>4.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>163</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>4576</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160327</td>\n      <td>5900</td>\n    </tr>\n    <tr>\n      <th>149996</th>\n      <td>149996</td>\n      <td>184535</td>\n      <td>20091102</td>\n      <td>116.0</td>\n      <td>11</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>125</td>\n      <td>10.0</td>\n      <td>0.0</td>\n      <td>2826</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160312</td>\n      <td>9500</td>\n    </tr>\n    <tr>\n      <th>149997</th>\n      <td>149997</td>\n      <td>147587</td>\n      <td>20101003</td>\n      <td>60.0</td>\n      <td>11</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>90</td>\n      <td>6.0</td>\n      <td>0.0</td>\n      <td>3302</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160328</td>\n      <td>7500</td>\n    </tr>\n    <tr>\n      <th>149998</th>\n      <td>149998</td>\n      <td>45907</td>\n      <td>20060312</td>\n      <td>34.0</td>\n      <td>10</td>\n      <td>3.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>156</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>1877</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160401</td>\n      <td>4999</td>\n    </tr>\n    <tr>\n      <th>149999</th>\n      <td>149999</td>\n      <td>177672</td>\n      <td>19990204</td>\n      <td>19.0</td>\n      <td>28</td>\n      <td>6.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>193</td>\n      <td>12.5</td>\n      <td>0.0</td>\n      <td>235</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160305</td>\n      <td>4700</td>\n    </tr>\n  </tbody>\n</table>\n<p>135884 rows × 16 columns</p>\n</div>"
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=data_csv.loc[data_csv.notna().all(1)] #还是采用loc[]的方式好理解得多啊\n",
    "df"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### 再尝试用dropna()\n",
    "\n",
    "重要参数：`subset`。删除行的指定列如果有nan的情况，默认会删除任何有nan值的行。的确感觉用`dropna()`要简洁很多"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": "        SaleID    name   regDate  model  brand  bodyType  fuelType  gearbox  \\\n0            0     736  20040402   30.0      6       1.0       0.0      0.0   \n1            1    2262  20030301   40.0      1       2.0       0.0      0.0   \n2            2   14874  20040403  115.0     15       1.0       0.0      0.0   \n3            3   71865  19960908  109.0     10       0.0       0.0      1.0   \n4            4  111080  20120103  110.0      5       1.0       0.0      0.0   \n...        ...     ...       ...    ...    ...       ...       ...      ...   \n149995  149995  163978  20000607  121.0     10       4.0       0.0      1.0   \n149996  149996  184535  20091102  116.0     11       0.0       0.0      0.0   \n149997  149997  147587  20101003   60.0     11       1.0       1.0      0.0   \n149998  149998   45907  20060312   34.0     10       3.0       1.0      0.0   \n149999  149999  177672  19990204   19.0     28       6.0       0.0      1.0   \n\n        power  kilometer notRepairedDamage  regionCode  seller  offerType  \\\n0          60       12.5               0.0        1046       0          0   \n1           0       15.0                 -        4366       0          0   \n2         163       12.5               0.0        2806       0          0   \n3         193       15.0               0.0         434       0          0   \n4          68        5.0               0.0        6977       0          0   \n...       ...        ...               ...         ...     ...        ...   \n149995    163       15.0               0.0        4576       0          0   \n149996    125       10.0               0.0        2826       0          0   \n149997     90        6.0               0.0        3302       0          0   \n149998    156       15.0               0.0        1877       0          0   \n149999    193       12.5               0.0         235       0          0   \n\n        creatDate  price  \n0        20160404   1850  \n1        20160309   3600  \n2        20160402   6222  \n3        20160312   2400  \n4        20160313   5200  \n...           ...    ...  \n149995   20160327   5900  \n149996   20160312   9500  \n149997   20160328   7500  \n149998   20160401   4999  \n149999   20160305   4700  \n\n[149999 rows x 16 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>SaleID</th>\n      <th>name</th>\n      <th>regDate</th>\n      <th>model</th>\n      <th>brand</th>\n      <th>bodyType</th>\n      <th>fuelType</th>\n      <th>gearbox</th>\n      <th>power</th>\n      <th>kilometer</th>\n      <th>notRepairedDamage</th>\n      <th>regionCode</th>\n      <th>seller</th>\n      <th>offerType</th>\n      <th>creatDate</th>\n      <th>price</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>736</td>\n      <td>20040402</td>\n      <td>30.0</td>\n      <td>6</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>60</td>\n      <td>12.5</td>\n      <td>0.0</td>\n      <td>1046</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160404</td>\n      <td>1850</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>2262</td>\n      <td>20030301</td>\n      <td>40.0</td>\n      <td>1</td>\n      <td>2.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>-</td>\n      <td>4366</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160309</td>\n      <td>3600</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>2</td>\n      <td>14874</td>\n      <td>20040403</td>\n      <td>115.0</td>\n      <td>15</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>163</td>\n      <td>12.5</td>\n      <td>0.0</td>\n      <td>2806</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160402</td>\n      <td>6222</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3</td>\n      <td>71865</td>\n      <td>19960908</td>\n      <td>109.0</td>\n      <td>10</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>193</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>434</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160312</td>\n      <td>2400</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>4</td>\n      <td>111080</td>\n      <td>20120103</td>\n      <td>110.0</td>\n      <td>5</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>68</td>\n      <td>5.0</td>\n      <td>0.0</td>\n      <td>6977</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160313</td>\n      <td>5200</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>149995</th>\n      <td>149995</td>\n      <td>163978</td>\n      <td>20000607</td>\n      <td>121.0</td>\n      <td>10</td>\n      <td>4.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>163</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>4576</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160327</td>\n      <td>5900</td>\n    </tr>\n    <tr>\n      <th>149996</th>\n      <td>149996</td>\n      <td>184535</td>\n      <td>20091102</td>\n      <td>116.0</td>\n      <td>11</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>0.0</td>\n      <td>125</td>\n      <td>10.0</td>\n      <td>0.0</td>\n      <td>2826</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160312</td>\n      <td>9500</td>\n    </tr>\n    <tr>\n      <th>149997</th>\n      <td>149997</td>\n      <td>147587</td>\n      <td>20101003</td>\n      <td>60.0</td>\n      <td>11</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>90</td>\n      <td>6.0</td>\n      <td>0.0</td>\n      <td>3302</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160328</td>\n      <td>7500</td>\n    </tr>\n    <tr>\n      <th>149998</th>\n      <td>149998</td>\n      <td>45907</td>\n      <td>20060312</td>\n      <td>34.0</td>\n      <td>10</td>\n      <td>3.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>156</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>1877</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160401</td>\n      <td>4999</td>\n    </tr>\n    <tr>\n      <th>149999</th>\n      <td>149999</td>\n      <td>177672</td>\n      <td>19990204</td>\n      <td>19.0</td>\n      <td>28</td>\n      <td>6.0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>193</td>\n      <td>12.5</td>\n      <td>0.0</td>\n      <td>235</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160305</td>\n      <td>4700</td>\n    </tr>\n  </tbody>\n</table>\n<p>149999 rows × 16 columns</p>\n</div>"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_csv.dropna(subset=['model'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 缺失值填充`fillna()`\n",
    "`ctrl+Q`一波其实讲解得很清楚了"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "data": {
      "text/plain": "        SaleID    name   regDate  model  brand  bodyType  fuelType  gearbox  \\\n14          14    1896  20070009    1.0      0       NaN       NaN      0.0   \n21          21   12784  20021009    8.0      0       0.0       NaN      0.0   \n42          42   20694  19960009    0.0      0       4.0       NaN      0.0   \n45          45    8893  20010306   16.0     13       1.0       NaN      0.0   \n98          98   31752  20020011    1.0      1       2.0       NaN      1.0   \n...        ...     ...       ...    ...    ...       ...       ...      ...   \n149937  149937   93471  20071110    8.0      0       0.0       1.0      0.0   \n149957  149957    3075  20000312   73.0     14       0.0       1.0      0.0   \n149970  149970  102173  19970406   48.0     14       1.0       1.0      0.0   \n149972  149972  183896  20050004   41.0      6       1.0       1.0      0.0   \n149973  149973   17265  20051104   49.0      1       0.0       1.0      0.0   \n\n        power  kilometer notRepairedDamage  regionCode  seller  offerType  \\\n14          0       15.0                 -        3972       0          0   \n21        116       15.0                 -        1278       0          0   \n42         90       15.0               0.0        4825       0          0   \n45         60       15.0               0.0        1326       0          0   \n98          0       15.0               0.0        6792       0          0   \n...       ...        ...               ...         ...     ...        ...   \n149937      0       15.0               0.0        1128       0          0   \n149957    130       15.0                 -        4439       0          0   \n149970     45       15.0               0.0        2159       0          0   \n149972      0       15.0                 -        7078       0          0   \n149973      0       15.0               0.0        4379       0          0   \n\n        creatDate  price  \n14       20160402   6900  \n21       20160403   2800  \n42       20160328   1600  \n45       20160330    850  \n98       20160310   3000  \n...           ...    ...  \n149937   20160326   3100  \n149957   20160402    600  \n149970   20160407    900  \n149972   20160326    999  \n149973   20160316   3500  \n\n[14116 rows x 16 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>SaleID</th>\n      <th>name</th>\n      <th>regDate</th>\n      <th>model</th>\n      <th>brand</th>\n      <th>bodyType</th>\n      <th>fuelType</th>\n      <th>gearbox</th>\n      <th>power</th>\n      <th>kilometer</th>\n      <th>notRepairedDamage</th>\n      <th>regionCode</th>\n      <th>seller</th>\n      <th>offerType</th>\n      <th>creatDate</th>\n      <th>price</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>14</th>\n      <td>14</td>\n      <td>1896</td>\n      <td>20070009</td>\n      <td>1.0</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>-</td>\n      <td>3972</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160402</td>\n      <td>6900</td>\n    </tr>\n    <tr>\n      <th>21</th>\n      <td>21</td>\n      <td>12784</td>\n      <td>20021009</td>\n      <td>8.0</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>NaN</td>\n      <td>0.0</td>\n      <td>116</td>\n      <td>15.0</td>\n      <td>-</td>\n      <td>1278</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160403</td>\n      <td>2800</td>\n    </tr>\n    <tr>\n      <th>42</th>\n      <td>42</td>\n      <td>20694</td>\n      <td>19960009</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>4.0</td>\n      <td>NaN</td>\n      <td>0.0</td>\n      <td>90</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>4825</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160328</td>\n      <td>1600</td>\n    </tr>\n    <tr>\n      <th>45</th>\n      <td>45</td>\n      <td>8893</td>\n      <td>20010306</td>\n      <td>16.0</td>\n      <td>13</td>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>0.0</td>\n      <td>60</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>1326</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160330</td>\n      <td>850</td>\n    </tr>\n    <tr>\n      <th>98</th>\n      <td>98</td>\n      <td>31752</td>\n      <td>20020011</td>\n      <td>1.0</td>\n      <td>1</td>\n      <td>2.0</td>\n      <td>NaN</td>\n      <td>1.0</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>6792</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160310</td>\n      <td>3000</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>149937</th>\n      <td>149937</td>\n      <td>93471</td>\n      <td>20071110</td>\n      <td>8.0</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>1128</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160326</td>\n      <td>3100</td>\n    </tr>\n    <tr>\n      <th>149957</th>\n      <td>149957</td>\n      <td>3075</td>\n      <td>20000312</td>\n      <td>73.0</td>\n      <td>14</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>130</td>\n      <td>15.0</td>\n      <td>-</td>\n      <td>4439</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160402</td>\n      <td>600</td>\n    </tr>\n    <tr>\n      <th>149970</th>\n      <td>149970</td>\n      <td>102173</td>\n      <td>19970406</td>\n      <td>48.0</td>\n      <td>14</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>45</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>2159</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160407</td>\n      <td>900</td>\n    </tr>\n    <tr>\n      <th>149972</th>\n      <td>149972</td>\n      <td>183896</td>\n      <td>20050004</td>\n      <td>41.0</td>\n      <td>6</td>\n      <td>1.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>-</td>\n      <td>7078</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160326</td>\n      <td>999</td>\n    </tr>\n    <tr>\n      <th>149973</th>\n      <td>149973</td>\n      <td>17265</td>\n      <td>20051104</td>\n      <td>49.0</td>\n      <td>1</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>4379</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160316</td>\n      <td>3500</td>\n    </tr>\n  </tbody>\n</table>\n<p>14116 rows × 16 columns</p>\n</div>"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_csv.loc[data_csv.isnull().any(1)].fillna(method='ffill')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "data": {
      "text/plain": "        SaleID    name   regDate  model  brand  bodyType  fuelType  gearbox  \\\n14          14    1896  20070009    1.0      0       NaN      -1.0      0.0   \n21          21   12784  20021009    8.0      0       0.0      -1.0      NaN   \n42          42   20694  19960009    0.0      0       4.0      -1.0      0.0   \n45          45    8893  20010306   16.0     13       1.0      -1.0      0.0   \n98          98   31752  20020011    1.0      1       2.0      -1.0      1.0   \n...        ...     ...       ...    ...    ...       ...       ...      ...   \n149937  149937   93471  20071110    8.0      0       0.0       1.0      NaN   \n149957  149957    3075  20000312   73.0     14       NaN      -1.0      0.0   \n149970  149970  102173  19970406   48.0     14       1.0      -1.0      0.0   \n149972  149972  183896  20050004   41.0      6       NaN      -1.0      NaN   \n149973  149973   17265  20051104   49.0      1       0.0       1.0      NaN   \n\n        power  kilometer notRepairedDamage  regionCode  seller  offerType  \\\n14          0       15.0                 -        3972       0          0   \n21        116       15.0                 -        1278       0          0   \n42         90       15.0               0.0        4825       0          0   \n45         60       15.0               0.0        1326       0          0   \n98          0       15.0               0.0        6792       0          0   \n...       ...        ...               ...         ...     ...        ...   \n149937      0       15.0               0.0        1128       0          0   \n149957    130       15.0                 -        4439       0          0   \n149970     45       15.0               0.0        2159       0          0   \n149972      0       15.0                 -        7078       0          0   \n149973      0       15.0               0.0        4379       0          0   \n\n        creatDate  price  \n14       20160402   6900  \n21       20160403   2800  \n42       20160328   1600  \n45       20160330    850  \n98       20160310   3000  \n...           ...    ...  \n149937   20160326   3100  \n149957   20160402    600  \n149970   20160407    900  \n149972   20160326    999  \n149973   20160316   3500  \n\n[14116 rows x 16 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>SaleID</th>\n      <th>name</th>\n      <th>regDate</th>\n      <th>model</th>\n      <th>brand</th>\n      <th>bodyType</th>\n      <th>fuelType</th>\n      <th>gearbox</th>\n      <th>power</th>\n      <th>kilometer</th>\n      <th>notRepairedDamage</th>\n      <th>regionCode</th>\n      <th>seller</th>\n      <th>offerType</th>\n      <th>creatDate</th>\n      <th>price</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>14</th>\n      <td>14</td>\n      <td>1896</td>\n      <td>20070009</td>\n      <td>1.0</td>\n      <td>0</td>\n      <td>NaN</td>\n      <td>-1.0</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>-</td>\n      <td>3972</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160402</td>\n      <td>6900</td>\n    </tr>\n    <tr>\n      <th>21</th>\n      <td>21</td>\n      <td>12784</td>\n      <td>20021009</td>\n      <td>8.0</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>-1.0</td>\n      <td>NaN</td>\n      <td>116</td>\n      <td>15.0</td>\n      <td>-</td>\n      <td>1278</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160403</td>\n      <td>2800</td>\n    </tr>\n    <tr>\n      <th>42</th>\n      <td>42</td>\n      <td>20694</td>\n      <td>19960009</td>\n      <td>0.0</td>\n      <td>0</td>\n      <td>4.0</td>\n      <td>-1.0</td>\n      <td>0.0</td>\n      <td>90</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>4825</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160328</td>\n      <td>1600</td>\n    </tr>\n    <tr>\n      <th>45</th>\n      <td>45</td>\n      <td>8893</td>\n      <td>20010306</td>\n      <td>16.0</td>\n      <td>13</td>\n      <td>1.0</td>\n      <td>-1.0</td>\n      <td>0.0</td>\n      <td>60</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>1326</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160330</td>\n      <td>850</td>\n    </tr>\n    <tr>\n      <th>98</th>\n      <td>98</td>\n      <td>31752</td>\n      <td>20020011</td>\n      <td>1.0</td>\n      <td>1</td>\n      <td>2.0</td>\n      <td>-1.0</td>\n      <td>1.0</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>6792</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160310</td>\n      <td>3000</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>149937</th>\n      <td>149937</td>\n      <td>93471</td>\n      <td>20071110</td>\n      <td>8.0</td>\n      <td>0</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>1128</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160326</td>\n      <td>3100</td>\n    </tr>\n    <tr>\n      <th>149957</th>\n      <td>149957</td>\n      <td>3075</td>\n      <td>20000312</td>\n      <td>73.0</td>\n      <td>14</td>\n      <td>NaN</td>\n      <td>-1.0</td>\n      <td>0.0</td>\n      <td>130</td>\n      <td>15.0</td>\n      <td>-</td>\n      <td>4439</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160402</td>\n      <td>600</td>\n    </tr>\n    <tr>\n      <th>149970</th>\n      <td>149970</td>\n      <td>102173</td>\n      <td>19970406</td>\n      <td>48.0</td>\n      <td>14</td>\n      <td>1.0</td>\n      <td>-1.0</td>\n      <td>0.0</td>\n      <td>45</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>2159</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160407</td>\n      <td>900</td>\n    </tr>\n    <tr>\n      <th>149972</th>\n      <td>149972</td>\n      <td>183896</td>\n      <td>20050004</td>\n      <td>41.0</td>\n      <td>6</td>\n      <td>NaN</td>\n      <td>-1.0</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>-</td>\n      <td>7078</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160326</td>\n      <td>999</td>\n    </tr>\n    <tr>\n      <th>149973</th>\n      <td>149973</td>\n      <td>17265</td>\n      <td>20051104</td>\n      <td>49.0</td>\n      <td>1</td>\n      <td>0.0</td>\n      <td>1.0</td>\n      <td>NaN</td>\n      <td>0</td>\n      <td>15.0</td>\n      <td>0.0</td>\n      <td>4379</td>\n      <td>0</td>\n      <td>0</td>\n      <td>20160316</td>\n      <td>3500</td>\n    </tr>\n  </tbody>\n</table>\n<p>14116 rows × 16 columns</p>\n</div>"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_csv.loc[data_csv.isnull().any(1)].fillna(value={'fuelType':-1})"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### 练习"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Ex1：缺失值与类别的相关性检验\n",
    "打扰了，我先试试Ex2吧。这个Ex1我看题都看了半天也没看懂"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "outputs": [],
   "source": [
    "df_ex1 = pd.read_csv('./data/missing_chi.csv')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "outputs": [
    {
     "data": {
      "text/plain": "X_1    0.855\nX_2    0.894\ny      0.000\ndtype: float64"
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ex1.isna().mean(axis=0)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "data": {
      "text/plain": "0    918\n1     82\nName: y, dtype: int64"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ex1['y'].value_counts()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "#### Ex2：用回归模型解决分类问题"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "第一小问"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "data": {
      "text/plain": "    X1   X2 Color\n0 -2.5  2.8  Blue\n1 -1.5  1.8  Blue\n2 -0.8  2.8  Blue\n3 -0.3  0.8  Blue\n4  1.1  2.1  Blue",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>X1</th>\n      <th>X2</th>\n      <th>Color</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-2.5</td>\n      <td>2.8</td>\n      <td>Blue</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-1.5</td>\n      <td>1.8</td>\n      <td>Blue</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>-0.8</td>\n      <td>2.8</td>\n      <td>Blue</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>-0.3</td>\n      <td>0.8</td>\n      <td>Blue</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1.1</td>\n      <td>2.1</td>\n      <td>Blue</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ex2 = pd.read_excel('./data/color.xlsx')\n",
    "df_ex2.head()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 23 entries, 0 to 22\n",
      "Data columns (total 3 columns):\n",
      " #   Column  Non-Null Count  Dtype  \n",
      "---  ------  --------------  -----  \n",
      " 0   X1      23 non-null     float64\n",
      " 1   X2      23 non-null     float64\n",
      " 2   Color   23 non-null     object \n",
      "dtypes: float64(2), object(1)\n",
      "memory usage: 680.0+ bytes\n"
     ]
    }
   ],
   "source": [
    "df_ex2.info()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "data": {
      "text/plain": "X1       0.0\nX2       0.0\nColor    0.0\ndtype: float64"
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ex2.isna().mean(axis=0) # 没有缺失值"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [
    {
     "data": {
      "text/plain": "Yellow    8\nGreen     8\nBlue      7\nName: Color, dtype: int64"
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ex2['Color'].value_counts() # 表明有三种类别的颜色，需要分别替换成0，1，2"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "### dummies()\n",
    "这里学到了一个非常先进的离散数据处理方法，参考的文章是：[python中get_dummies实践](https://blog.csdn.net/dongyanwen6036/article/details/78555163)：\n",
    ">1、离散特征的取值之间**没有大小的意义**，比如color：[red,blue],那么就使用one-hot编码\n",
    ">\n",
    ">2、离散特征的取值**有大小的意义**，比如size:[X,XL,XXL],那么就使用数值的映射{X:1,XL:2,XXL:3}。说明：对于有大小意义的离散特征，直接使用映射就可以了，{'XL':3,'L':2,'M':1}\n",
    "\n",
    "所以我之前对Color这个属性的处理方式是错误的，我直接将三种颜色进行了`replace()`，但是很明显这三种颜色是没有大小意义的，应该使用one-hot编码。"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [],
   "source": [
    "df_dummies = pd.get_dummies(df_ex2['Color'])\n",
    "df_ex2 = df_ex2.join(df_dummies)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "data": {
      "text/plain": "     X1   X2   Color  Blue  Green  Yellow\n0  -2.5  2.8    Blue     1      0       0\n1  -1.5  1.8    Blue     1      0       0\n2  -0.8  2.8    Blue     1      0       0\n3  -0.3  0.8    Blue     1      0       0\n4   1.1  2.1    Blue     1      0       0\n5   1.6  3.0    Blue     1      0       0\n6   3.2  3.2    Blue     1      0       0\n7   0.0 -2.2  Yellow     0      0       1\n8   0.0  2.0  Yellow     0      0       1\n9   2.2 -0.8  Yellow     0      0       1\n10  2.5  0.3  Yellow     0      0       1\n11  3.9  0.3  Yellow     0      0       1\n12  3.9  2.9  Yellow     0      0       1\n13  4.1 -1.3  Yellow     0      0       1\n14  4.3  1.8  Yellow     0      0       1\n15 -3.2  1.0   Green     0      1       0\n16 -2.7 -0.5   Green     0      1       0\n17 -2.0 -0.4   Green     0      1       0\n18 -1.9 -2.0   Green     0      1       0\n19 -1.5  0.5   Green     0      1       0\n20 -1.0  1.5   Green     0      1       0\n21  0.2 -0.7   Green     0      1       0\n22  1.5 -1.3   Green     0      1       0",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>X1</th>\n      <th>X2</th>\n      <th>Color</th>\n      <th>Blue</th>\n      <th>Green</th>\n      <th>Yellow</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>-2.5</td>\n      <td>2.8</td>\n      <td>Blue</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>-1.5</td>\n      <td>1.8</td>\n      <td>Blue</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>-0.8</td>\n      <td>2.8</td>\n      <td>Blue</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>-0.3</td>\n      <td>0.8</td>\n      <td>Blue</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1.1</td>\n      <td>2.1</td>\n      <td>Blue</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>1.6</td>\n      <td>3.0</td>\n      <td>Blue</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>6</th>\n      <td>3.2</td>\n      <td>3.2</td>\n      <td>Blue</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>0.0</td>\n      <td>-2.2</td>\n      <td>Yellow</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>8</th>\n      <td>0.0</td>\n      <td>2.0</td>\n      <td>Yellow</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>9</th>\n      <td>2.2</td>\n      <td>-0.8</td>\n      <td>Yellow</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>10</th>\n      <td>2.5</td>\n      <td>0.3</td>\n      <td>Yellow</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>11</th>\n      <td>3.9</td>\n      <td>0.3</td>\n      <td>Yellow</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>12</th>\n      <td>3.9</td>\n      <td>2.9</td>\n      <td>Yellow</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>13</th>\n      <td>4.1</td>\n      <td>-1.3</td>\n      <td>Yellow</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>14</th>\n      <td>4.3</td>\n      <td>1.8</td>\n      <td>Yellow</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>15</th>\n      <td>-3.2</td>\n      <td>1.0</td>\n      <td>Green</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>16</th>\n      <td>-2.7</td>\n      <td>-0.5</td>\n      <td>Green</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>17</th>\n      <td>-2.0</td>\n      <td>-0.4</td>\n      <td>Green</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>18</th>\n      <td>-1.9</td>\n      <td>-2.0</td>\n      <td>Green</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>19</th>\n      <td>-1.5</td>\n      <td>0.5</td>\n      <td>Green</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>20</th>\n      <td>-1.0</td>\n      <td>1.5</td>\n      <td>Green</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>21</th>\n      <td>0.2</td>\n      <td>-0.7</td>\n      <td>Green</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>22</th>\n      <td>1.5</td>\n      <td>-1.3</td>\n      <td>Green</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ex2"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "下面这段代码很精髓\n",
    "\n",
    "首先介绍一下KNN：借助K个最近训练样本的目标数值，对待测样本的回归值进行决策，即根据样本的相似度预测回归值。也就是说这个K值其实是可以不用设定的，就直接用默认值5也可以\n",
    "\n",
    "其实下面训练了三个回归模型，每个模型只预测一种颜色的可能性，然后应该取这三种模型预测值最大的作为最终预测颜色\n"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [
    {
     "data": {
      "text/plain": "[array([[0.16666667]]), array([[0.33333333]]), array([[0.5]])]"
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsRegressor,KNeighborsClassifier\n",
    "stack_list = []\n",
    "for col in df_dummies.columns:\n",
    "    clf = KNeighborsRegressor(n_neighbors=6)\n",
    "    # 比如这里，当col为Blue时，只有blue字段为1的行是正样本，其他都是负样本\n",
    "    clf.fit(df_ex2.iloc[:,:2], df_dummies[col])\n",
    "    res = clf.predict([[0.8, -0.2]]).reshape(-1,1)\n",
    "    stack_list.append(res)\n",
    "stack_list"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [
    {
     "data": {
      "text/plain": "0    2\ndtype: int64"
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_res = pd.Series(np.hstack(stack_list).argmax(1))\n",
    "code_res"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [
    {
     "data": {
      "text/plain": "'Yellow'"
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_dummies.columns[code_res[0]]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "第二小问"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "outputs": [
    {
     "data": {
      "text/plain": "        ID  Age Employment    Marital     Income  Gender  Hours\n0  1004641   38    Private  Unmarried   81838.00  Female     72\n1  1010229   35    Private     Absent   72099.00    Male     30\n2  1024587   32    Private   Divorced  154676.74    Male     40\n3  1038288   45    Private    Married   27743.82    Male     55\n4  1044221   60    Private    Married    7568.23    Male     40",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>ID</th>\n      <th>Age</th>\n      <th>Employment</th>\n      <th>Marital</th>\n      <th>Income</th>\n      <th>Gender</th>\n      <th>Hours</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1004641</td>\n      <td>38</td>\n      <td>Private</td>\n      <td>Unmarried</td>\n      <td>81838.00</td>\n      <td>Female</td>\n      <td>72</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1010229</td>\n      <td>35</td>\n      <td>Private</td>\n      <td>Absent</td>\n      <td>72099.00</td>\n      <td>Male</td>\n      <td>30</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1024587</td>\n      <td>32</td>\n      <td>Private</td>\n      <td>Divorced</td>\n      <td>154676.74</td>\n      <td>Male</td>\n      <td>40</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1038288</td>\n      <td>45</td>\n      <td>Private</td>\n      <td>Married</td>\n      <td>27743.82</td>\n      <td>Male</td>\n      <td>55</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1044221</td>\n      <td>60</td>\n      <td>Private</td>\n      <td>Married</td>\n      <td>7568.23</td>\n      <td>Male</td>\n      <td>40</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_audit = pd.read_csv('./data/audit.csv')\n",
    "data_audit.head() # 发现其实不需要ID这个字段作为预测维度"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "分析数据缺失情况，发现只有Employment有缺失，这也正是我们需要做的"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [
    {
     "data": {
      "text/plain": "ID            0.00\nAge           0.00\nEmployment    0.05\nMarital       0.00\nIncome        0.00\nGender        0.00\nHours         0.00\ndtype: float64"
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_audit.isna().mean(axis=0)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "outputs": [],
   "source": [
    "data_audit = data_audit.drop(columns=['ID'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "分析离散类型特征。主要工作是one-hot或者replace()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "outputs": [
    {
     "data": {
      "text/plain": "      Employment_Consultant  Employment_PSFederal  Employment_PSLocal  \\\n0                         0                     0                   0   \n1                         0                     0                   0   \n2                         0                     0                   0   \n3                         0                     0                   0   \n4                         0                     0                   0   \n...                     ...                   ...                 ...   \n1995                      0                     0                   0   \n1996                      1                     0                   0   \n1997                      0                     0                   0   \n1998                      0                     0                   0   \n1999                      0                     0                   0   \n\n      Employment_PSState  Employment_Private  Employment_SelfEmp  \\\n0                      0                   1                   0   \n1                      0                   1                   0   \n2                      0                   1                   0   \n3                      0                   1                   0   \n4                      0                   1                   0   \n...                  ...                 ...                 ...   \n1995                   0                   1                   0   \n1996                   0                   0                   0   \n1997                   0                   1                   0   \n1998                   0                   1                   0   \n1999                   0                   1                   0   \n\n      Employment_Unemployed  Employment_Volunteer  Marital_Absent  \\\n0                         0                     0               0   \n1                         0                     0               1   \n2                         0                     0               0   \n3                         0                     0               0   \n4                         0                     0               0   \n...                     ...                   ...             ...   \n1995                      0                     0               0   \n1996                      0                     0               0   \n1997                      0                     0               0   \n1998                      0                     0               0   \n1999                      0                     0               1   \n\n      Marital_Divorced  Marital_Married  Marital_Married-spouse-absent  \\\n0                    0                0                              0   \n1                    0                0                              0   \n2                    1                0                              0   \n3                    0                1                              0   \n4                    0                1                              0   \n...                ...              ...                            ...   \n1995                 0                1                              0   \n1996                 0                1                              0   \n1997                 0                1                              0   \n1998                 0                0                              0   \n1999                 0                0                              0   \n\n      Marital_Unmarried  Marital_Widowed  Gender_Female  Gender_Male  \n0                     1                0              1            0  \n1                     0                0              0            1  \n2                     0                0              0            1  \n3                     0                0              0            1  \n4                     0                0              0            1  \n...                 ...              ...            ...          ...  \n1995                  0                0              0            1  \n1996                  0                0              0            1  \n1997                  0                0              0            1  \n1998                  1                0              0            1  \n1999                  0                0              1            0  \n\n[2000 rows x 16 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Employment_Consultant</th>\n      <th>Employment_PSFederal</th>\n      <th>Employment_PSLocal</th>\n      <th>Employment_PSState</th>\n      <th>Employment_Private</th>\n      <th>Employment_SelfEmp</th>\n      <th>Employment_Unemployed</th>\n      <th>Employment_Volunteer</th>\n      <th>Marital_Absent</th>\n      <th>Marital_Divorced</th>\n      <th>Marital_Married</th>\n      <th>Marital_Married-spouse-absent</th>\n      <th>Marital_Unmarried</th>\n      <th>Marital_Widowed</th>\n      <th>Gender_Female</th>\n      <th>Gender_Male</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1995</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1996</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1997</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1998</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1999</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>2000 rows × 16 columns</p>\n</div>"
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res_df = data_audit.copy()\n",
    "\n",
    "separate_dummies = pd.get_dummies(data_audit[['Employment','Marital','Gender']])\n",
    "separate_dummies"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "分析连续类型特征。主要工作是归一化\n",
    "\n",
    "这个部分参考答案中用到了归一化，这个操作终于让我等到了！同时我也参考了一篇文章，里面简要提到了对特征归一化的必要性：[一文搞懂k近邻（k-NN）算法（一）](https://zhuanlan.zhihu.com/p/25994179)\n",
    "\n",
    "同时注意一下这里的`apply()`中输入的x是一个个的`Series`，总共有三个，分别为`Age,Income,Hours`，可以对他们取`max(),min()`等。"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "outputs": [
    {
     "data": {
      "text/plain": "           Age    Income     Hours\n0     0.287671  0.168997  0.724490\n1     0.246575  0.148735  0.295918\n2     0.205479  0.320539  0.397959\n3     0.383562  0.056453  0.551020\n4     0.589041  0.014477  0.397959\n...        ...       ...       ...\n1995  0.616438  0.048832  0.397959\n1996  0.246575  0.118356  0.397959\n1997  0.205479  0.062267  0.438776\n1998  0.232877  0.234715  0.448980\n1999  0.123288  0.289972  0.346939\n\n[2000 rows x 3 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Age</th>\n      <th>Income</th>\n      <th>Hours</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0.287671</td>\n      <td>0.168997</td>\n      <td>0.724490</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0.246575</td>\n      <td>0.148735</td>\n      <td>0.295918</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0.205479</td>\n      <td>0.320539</td>\n      <td>0.397959</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0.383562</td>\n      <td>0.056453</td>\n      <td>0.551020</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0.589041</td>\n      <td>0.014477</td>\n      <td>0.397959</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1995</th>\n      <td>0.616438</td>\n      <td>0.048832</td>\n      <td>0.397959</td>\n    </tr>\n    <tr>\n      <th>1996</th>\n      <td>0.246575</td>\n      <td>0.118356</td>\n      <td>0.397959</td>\n    </tr>\n    <tr>\n      <th>1997</th>\n      <td>0.205479</td>\n      <td>0.062267</td>\n      <td>0.438776</td>\n    </tr>\n    <tr>\n      <th>1998</th>\n      <td>0.232877</td>\n      <td>0.234715</td>\n      <td>0.448980</td>\n    </tr>\n    <tr>\n      <th>1999</th>\n      <td>0.123288</td>\n      <td>0.289972</td>\n      <td>0.346939</td>\n    </tr>\n  </tbody>\n</table>\n<p>2000 rows × 3 columns</p>\n</div>"
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "continuous_dummies = data_audit[['Age','Income','Hours']].apply(lambda x:(x-x.min())/(x.max()-x.min()))\n",
    "continuous_dummies"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "outputs": [
    {
     "data": {
      "text/plain": "      Employment_Consultant  Employment_PSFederal  Employment_PSLocal  \\\n0                         0                     0                   0   \n1                         0                     0                   0   \n2                         0                     0                   0   \n3                         0                     0                   0   \n4                         0                     0                   0   \n...                     ...                   ...                 ...   \n1995                      0                     0                   0   \n1996                      1                     0                   0   \n1997                      0                     0                   0   \n1998                      0                     0                   0   \n1999                      0                     0                   0   \n\n      Employment_PSState  Employment_Private  Employment_SelfEmp  \\\n0                      0                   1                   0   \n1                      0                   1                   0   \n2                      0                   1                   0   \n3                      0                   1                   0   \n4                      0                   1                   0   \n...                  ...                 ...                 ...   \n1995                   0                   1                   0   \n1996                   0                   0                   0   \n1997                   0                   1                   0   \n1998                   0                   1                   0   \n1999                   0                   1                   0   \n\n      Employment_Unemployed  Employment_Volunteer  Marital_Absent  \\\n0                         0                     0               0   \n1                         0                     0               1   \n2                         0                     0               0   \n3                         0                     0               0   \n4                         0                     0               0   \n...                     ...                   ...             ...   \n1995                      0                     0               0   \n1996                      0                     0               0   \n1997                      0                     0               0   \n1998                      0                     0               0   \n1999                      0                     0               1   \n\n      Marital_Divorced  Marital_Married  Marital_Married-spouse-absent  \\\n0                    0                0                              0   \n1                    0                0                              0   \n2                    1                0                              0   \n3                    0                1                              0   \n4                    0                1                              0   \n...                ...              ...                            ...   \n1995                 0                1                              0   \n1996                 0                1                              0   \n1997                 0                1                              0   \n1998                 0                0                              0   \n1999                 0                0                              0   \n\n      Marital_Unmarried  Marital_Widowed  Gender_Female  Gender_Male  \\\n0                     1                0              1            0   \n1                     0                0              0            1   \n2                     0                0              0            1   \n3                     0                0              0            1   \n4                     0                0              0            1   \n...                 ...              ...            ...          ...   \n1995                  0                0              0            1   \n1996                  0                0              0            1   \n1997                  0                0              0            1   \n1998                  1                0              0            1   \n1999                  0                0              1            0   \n\n           Age    Income     Hours  \n0     0.287671  0.168997  0.724490  \n1     0.246575  0.148735  0.295918  \n2     0.205479  0.320539  0.397959  \n3     0.383562  0.056453  0.551020  \n4     0.589041  0.014477  0.397959  \n...        ...       ...       ...  \n1995  0.616438  0.048832  0.397959  \n1996  0.246575  0.118356  0.397959  \n1997  0.205479  0.062267  0.438776  \n1998  0.232877  0.234715  0.448980  \n1999  0.123288  0.289972  0.346939  \n\n[1900 rows x 19 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Employment_Consultant</th>\n      <th>Employment_PSFederal</th>\n      <th>Employment_PSLocal</th>\n      <th>Employment_PSState</th>\n      <th>Employment_Private</th>\n      <th>Employment_SelfEmp</th>\n      <th>Employment_Unemployed</th>\n      <th>Employment_Volunteer</th>\n      <th>Marital_Absent</th>\n      <th>Marital_Divorced</th>\n      <th>Marital_Married</th>\n      <th>Marital_Married-spouse-absent</th>\n      <th>Marital_Unmarried</th>\n      <th>Marital_Widowed</th>\n      <th>Gender_Female</th>\n      <th>Gender_Male</th>\n      <th>Age</th>\n      <th>Income</th>\n      <th>Hours</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0.287671</td>\n      <td>0.168997</td>\n      <td>0.724490</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0.246575</td>\n      <td>0.148735</td>\n      <td>0.295918</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0.205479</td>\n      <td>0.320539</td>\n      <td>0.397959</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0.383562</td>\n      <td>0.056453</td>\n      <td>0.551020</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0.589041</td>\n      <td>0.014477</td>\n      <td>0.397959</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1995</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0.616438</td>\n      <td>0.048832</td>\n      <td>0.397959</td>\n    </tr>\n    <tr>\n      <th>1996</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0.246575</td>\n      <td>0.118356</td>\n      <td>0.397959</td>\n    </tr>\n    <tr>\n      <th>1997</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0.205479</td>\n      <td>0.062267</td>\n      <td>0.438776</td>\n    </tr>\n    <tr>\n      <th>1998</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0.232877</td>\n      <td>0.234715</td>\n      <td>0.448980</td>\n    </tr>\n    <tr>\n      <th>1999</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0.123288</td>\n      <td>0.289972</td>\n      <td>0.346939</td>\n    </tr>\n  </tbody>\n</table>\n<p>1900 rows × 19 columns</p>\n</div>"
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.concat([separate_dummies,continuous_dummies],axis=1)  # 关注这里的axis参数\n",
    "X_train = df.loc[data_audit.notna().all(1)]\n",
    "X_test = df.loc[data_audit.isna().any(1)]\n",
    "X_train"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "outputs": [
    {
     "data": {
      "text/plain": "0          Private\n1          Private\n2          Private\n3          Private\n4          Private\n           ...    \n1995       Private\n1996    Consultant\n1997       Private\n1998       Private\n1999       Private\nName: Employment, Length: 1900, dtype: object"
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target = data_audit[data_audit['Employment'].notna()]['Employment']\n",
    "target"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [
    {
     "data": {
      "text/plain": "      Consultant  PSFederal  PSLocal  PSState  Private  SelfEmp  Unemployed  \\\n0              0          0        0        0        1        0           0   \n1              0          0        0        0        1        0           0   \n2              0          0        0        0        1        0           0   \n3              0          0        0        0        1        0           0   \n4              0          0        0        0        1        0           0   \n...          ...        ...      ...      ...      ...      ...         ...   \n1995           0          0        0        0        1        0           0   \n1996           1          0        0        0        0        0           0   \n1997           0          0        0        0        1        0           0   \n1998           0          0        0        0        1        0           0   \n1999           0          0        0        0        1        0           0   \n\n      Volunteer  \n0             0  \n1             0  \n2             0  \n3             0  \n4             0  \n...         ...  \n1995          0  \n1996          0  \n1997          0  \n1998          0  \n1999          0  \n\n[1900 rows x 8 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Consultant</th>\n      <th>PSFederal</th>\n      <th>PSLocal</th>\n      <th>PSState</th>\n      <th>Private</th>\n      <th>SelfEmp</th>\n      <th>Unemployed</th>\n      <th>Volunteer</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>1995</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1996</th>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1997</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1998</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1999</th>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n      <td>1</td>\n      <td>0</td>\n      <td>0</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>1900 rows × 8 columns</p>\n</div>"
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_dummies2 = pd.get_dummies(target)\n",
    "df_dummies2"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "outputs": [
    {
     "data": {
      "text/plain": "0     2\n1     0\n2     4\n3     4\n4     4\n     ..\n95    4\n96    4\n97    4\n98    4\n99    4\nLength: 100, dtype: int64"
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stack_list=[]\n",
    "for col in df_dummies2.columns:\n",
    "    clf = KNeighborsRegressor(n_neighbors=6)\n",
    "    clf.fit(X_train,df_dummies2[col])\n",
    "    res = clf.predict(X_test).reshape(-1,1)\n",
    "    stack_list.append(res)\n",
    "\n",
    "code_res = pd.Series(np.hstack(stack_list).argmax(1))\n",
    "code_res\n",
    "# 接下来就是填充"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "language": "python",
   "display_name": "Python 3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}